﻿//************************************************************************************************
// Copyright © 2012 Steven M. Cohn. All Rights Reserved.
//
//************************************************************************************************

namespace Cookie.Baker
{
	using System;
	using System.IO;
	using System.Text;
	using System.Collections.Generic;
	using Microsoft.VisualStudio.TestTools.UnitTesting;
	using CookieChipper;


	/*
	 * This is a test of the information found in the whitepaper:
	 * 
	 *   MSIE Cache File (index.dat) format specification, Analysis of the index.dat file format
	 *   By Joachim Metz <joachim.metz@gmail.com>
	 */

	#region CacheFlags
	[Flags]
	public enum CacheFlags
	{
		/// <summary>
		/// normal cache entry; can be deleted to recover space for new entries.
		/// </summary>
		Normal = 0x00000001,

		Stable = 0x00000002,

		/// <summary>
		/// sticky cache entry that is exempt from scavenging for the amount of time specified
		/// by release (exempt) delta. The default value set by the function CommitUrlCacheEntry
		/// is one day. Has extended flags (unknown value at offset 40): 0x00004008?
		/// </summary>
		Sticky = 0x00000004,

		/// <summary>
		/// Cache entry file that has been edited externally. This cache entry type is exempt
		/// from scavenging. Set for iecompat: and ietld:
		/// </summary>
		Edited = 0x00000008,

		/// <summary>
		/// Not currently implemented.
		/// </summary>
		TrackOffline = 0x00000010,

		/// <summary>
		/// Not currently implemented.
		/// </summary>
		TrackOnline = 0x00000020,

		/// <summary>
		///  Is cached/cache-able? Not set if header contains “Pragma: no-cache” or
		/// other cache related headers are present
		/// </summary>
		Cacheable = 0x00000040,

		/// <summary>
		///  HTTP request method, 0 => GET, 1 => POST
		/// </summary>
		Post = 0x00001000,

		/// <summary>
		/// Partial response cache entry.
		/// </summary>
		Sparse = 0x00010000,

		/// <summary>
		/// OLE Control Extension (OCX) cache entry.  OCX is a predecessor of ActiveX
		/// Set for PrivacIE:
		/// </summary>
		OCX = 0x00020000,

		/// <summary>
		/// Cookie cache entry.
		/// </summary>
		Cookie = 0x00100000,

		/// <summary>
		/// Visited link cache entry.
		/// </summary>
		History = 0x00200000
	}
	#endregion CacheFlags

	[TestClass]
	public class MMFDemo
	{
		private const int BlockSize = 128;
		private const string DateFormat = "MM/dd/yyyy h:mm tt";


		[TestMethod]
		public void ExamineCache ()
		{
			//string path = Path.Combine(
			//	Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
			//	@"Temp\Cookies\index.dat");

			string path = Path.Combine(CookieCacheProvider.GetDirectoryPath(), "index.dat");

			using (FileStream stream = File.Open(
					path, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite))
			{
				byte[] buffer;
				string signature;
				int number;
				int size;

				List<int> lookup = new List<int>();
				List<int> redirects = new List<int>();


				/*
				 * HEADER LAYOUT:
				 * 
				 * OFFSET	SIZE	DESCRIPTION
				 * 0		28		The signature and version string (Client UrlCache MMF Ver #.#\x00)
				 *					ASCII string with a NUL-character, # chars specify the version
				 * 28		4		The file size
				 * 32		4		The file offset to the first hash table (multple of 128 and >= 0x4000)
				 * 36		4		The total number of blocks
				 * 40		4		The number of allocated blocks
				 * 44		4		Unknown, empty value (0)
				 * 48		4		The cache size (quota) limit of the container, num of bytes
				 * 52		4		Unknown, empty value (0)
				 * 56		4		The cache size of the container, num of bytes
				 * 60		4		Unknown, empty value (0)
				 * 64		4		The non-releasable cache size of the container, num of bytes;
				 *					the size of the cache container exempt from scavenging
				 * 68		4		Unknown, empty value (0)
				 */

				size = 72;
				buffer = new byte[size];
				if (stream.Read(buffer, 0, size) == size)
				{
					signature = ASCIIEncoding.ASCII.GetString(buffer, 0, 28);
					Console.WriteLine("Version................: [" + signature + "]");

					number = BitConverter.ToInt32(buffer, 28);
					Console.WriteLine("File size..............: "
						+ number + " bytes (" + (number / 1024) + "KB)");

					number = BitConverter.ToInt32(buffer, 32);
					Console.WriteLine("First hash table offset: 0x"
						+ number.ToString("X") + " (" + number + " bytes)");

					number = BitConverter.ToInt32(buffer, 36);
					Console.WriteLine("Total blocks...........: "
						+ number + " (" + ((number * BlockSize) / 1024) + "KB)");

					number = BitConverter.ToInt32(buffer, 40);
					Console.WriteLine("Allocated blocks.......: "
						+ number + " (" + ((number * BlockSize) / 1024) + "KB)");

					number = BitConverter.ToInt32(buffer, 48);
					Console.WriteLine("Cache quota............: "
						+ number + " bytes (" + (number / BlockSize) + " blocks, " + (number / 1024) + "KB)");

					number = BitConverter.ToInt32(buffer, 56);
					Console.WriteLine("Cache size.............: "
						+ number + " bytes (" + (number / BlockSize) + " blocks, " + (number / 1024) + "KB)");

					number = BitConverter.ToInt32(buffer, 64);
					Console.WriteLine("Non-releasable size....: " + number + " bytes");
				}
				else
				{
					Console.WriteLine("could not read entire header");
					Assert.Fail();
				}

				/*
				 * CACHE DIRECTORY TABLE LAYOUT:
				 * 
				 * OFFSET	SIZE	DESCRIPTION
				 * 72		4		Number of cache directory entries (max 32?)
				 * 76		?		Cache directory entries...
				 * 
				 * DIRECTORY ENTRY LAYOUT:
				 * 
				 * 0		4		The number of cached files in the directory
				 * 4		8		Cache directory name, ASCII string without a NUL-character
				 */

				Assert.AreEqual(stream.Position, 72);

				size = 4;
				buffer = new byte[size];
				if (stream.Read(buffer, 0, size) == size)
				{
					Console.WriteLine();
					Console.WriteLine("DIRECTORY TABLE");

					number = BitConverter.ToInt32(buffer, 0);
					Console.WriteLine("Directory entries......: " + number);
				}
				else
				{
					Console.WriteLine("could not read directory header");
					Assert.Fail();
				}

				// TODO: dump directory entries

				/*
				 * ALLOCATION BITMAP LAYOUT: 0x250-0x4000
				 * 
				 * The allocation bitmap is situated at offset 592 (0x250) and continues to
				 * offset 16384 (0x4000) but only the number of bytes necessary for the file
				 * size is used. Every bit represents a block of 128 byte (0x80) starting at
				 * the (base) offset 0x4000. The bitmap is stored byte-wise where the LSB refers
				 * to the lowest offset e.g.
				 *		base offset................. : 0x4000
				 *		first allocation bitmap byte : 0xf0
				 *		unallocated range........... : 0x4000 – 0x4200
				 *		allocated range............. : 0x4200 - 0x4400
				 */

				while (stream.Position < 0x250)
				{
					stream.ReadByte();
				}

				Assert.AreEqual(stream.Position, 0x250);

				size = 16384 - 592; // 15792
				buffer = new byte[size];
				if (stream.Read(buffer, 0, size) == size)
				{
					Console.WriteLine();
					Console.WriteLine("ALLOCATION BITMAP");

					int block = 0;
					for (int i = 0; i < size; i++)
					{
						for (int b = 0; b < 8; b++, block++)
						{
							if ((buffer[i] & (1 << b)) > 0)
							{
								int offset = 0x4000 + (block * BlockSize);
								Console.WriteLine("... allocated block " + block + " at offset 0x" + offset.ToString("X") + " (" + offset + ")");
							}
						}
					}
				}
				else
				{
					Console.WriteLine("could not read allocation bitmap");
					Assert.Fail();
				}

				/*
				 * HASH TABLE LAYOUT: 0x4000-0x5000
				 * 
				 * OFFSET	SIZE	DESCRIPTION
				 * 0		4		The signature (“HASH”)
				 * 4		4		The number of blocks in hash table incl header (32 (0x20))
				 *					block size is 128: 32 x 128 = 4096 (0x1000)
				 *					The hash table entry data size is: 4096 – 12 = 4084
				 * 8		4		Next hash table record offset or 0 if this is the last
				 * 12		4		The sequence number; 0 identifies the first hash table record
				 * 
				 * HASH TABLE ENTRY LAYOUT:
				 * 
				 * 0		4		Record hash
				 * 4		4		Record offset; a multiple of 128 and >= 0x4000
				 *					If the record offset contains the same
				 *					value as the record hash, the value is unused
				 *					
				 * The record hash can contain different values:
				 * 
				 * VALUE			DESCRIPTION
				 * 0x#######0		Valid URL record
				 * 0x00000001		Invalid URL record?
				 * 0x00000003		unknown record, only seen as unused: with offset 0x00000003
				 * 0x#######5		REDR record
				 * 0x0badf00d		Hash table entry is uninitialized
				 * 0xdeadbeef		Hash table entry is uninitialized (MSIE 8)
				 * 
				 * The uninitialized hash table entries can occur in all hash table records not
				 * only the last one. Often the hash tables are only filled to offset 0xe00?
				 */

				Assert.AreEqual(stream.Position, 0x4000);

				size = 16;
				buffer = new byte[size];
				int nextHashTable = 0;

				do
				{
					if (nextHashTable > 0)
					{
						stream.Position = nextHashTable;
					}

					if (stream.Read(buffer, 0, size) == size)
					{
						Console.WriteLine();
						Console.WriteLine("HASH TABLE HEADER");

						signature = ASCIIEncoding.ASCII.GetString(buffer, 0, 4);
						Console.WriteLine("Signature..............: [" + signature + "]");

						number = BitConverter.ToInt32(buffer, 4);
						Console.WriteLine("Blocks incl header.....: "
							+ number + " (" + ((number * BlockSize) / 1024) + "KB)");

						nextHashTable = BitConverter.ToInt32(buffer, 8);
						Console.WriteLine("Next hash table offset.: 0x"
							+ nextHashTable.ToString("X") + " (" + nextHashTable + ")");

						number = BitConverter.ToInt32(buffer, 12);
						Console.WriteLine("Hash table seq number..: " + number);
					}
					else
					{
						Console.WriteLine("could not read hash table header");
						Assert.Fail();
					}

					// we've read 16 bytes of 0x1000 so...
					size = 0x1000 - 16;
					buffer = new byte[size];
					if (stream.Read(buffer, 0, size) == size)
					{
						Console.WriteLine();
						Console.WriteLine("HASH TABLE ENTRIES");

						uint valid = 0x0000000F;	// ^ this for low byte only
						uint UNKN = 0x00000003;		// exact
						uint REDR = 0x00000005;		// mask
						uint zero7 = 0x0badf00d;	// exact
						uint zero8 = 0xdeadbeef;	// exact

						int offset = 0;
						while (offset < size)
						{
							number = BitConverter.ToInt32(buffer, offset);

							if ((number & valid) == 0)
							{
								number = BitConverter.ToInt32(buffer, offset + 4);
								if (((number % BlockSize) == 0) && (number >= 0x4000))
								{
									lookup.Add(number);
									Console.WriteLine("valid record at offset 0x"
										+ number.ToString("X") + " (" + number + ")");
								}
							}
							else if (number == zero7)
							{
								Console.WriteLine("found zero7 badfood");
							}
							else if (number == zero8)
							{
								Console.WriteLine("found Zero8 deadbeef");
							}
							else if ((number & REDR) == REDR)
							{
								number = BitConverter.ToInt32(buffer, offset + 4);
								if (((number % BlockSize) == 0) && (number >= 0x4000))
								{
									redirects.Add(number);
									Console.WriteLine("found REDR at 0x"
										+ number.ToString("X") + " (" + number + ")");
								}
								else
								{
									Console.WriteLine("found REDR without address");
								}
							}
							else if (number == UNKN)
							{
								number = BitConverter.ToInt32(buffer, offset + 4);
								Console.WriteLine("found UNKN at " + number.ToString("X"));
							}
							else
							{
								Console.Write("? hash=" + number.ToString("X"));
								number = BitConverter.ToInt32(buffer, offset + 4);
								Console.WriteLine(" at " + number.ToString("X"));
							}

							offset += 8;
						}

						Console.WriteLine("Found " + lookup.Count + " valid entries");
					}
					else
					{
						Console.WriteLine("could not read hash table");
						Assert.Fail();
					}
				} while (nextHashTable != 0);

				// TODO: dump subsequent hash tables if exist

				/*
				 * URL RECORD LAYOUT
				 * 
				 * OFFSET	SIZE	DESCRIPTION
				 * 0		4		The signature (“URL\x20”)
				 * 4		4		The number of 128-byte blocks in URL record
				 * 8		8		The secondary filetime or 0 (last accessed for cookies)
				 * 16		8		The primary filetime or 0 (modified time for cookies)
				 * 24		4		Expiration date and time (or 0xFFFF for about: URI)
				 * 28		4		Unknown, empty
				 * 32		4		Cached file size in bytes
				 * 36		4		Upper part of a 64-bit cached file size in bytes
				 * 40		4		Unknown (0x00000004 for http:// ?)
				 * 44		4		The non-releasable time delta (864000 sec = 24hrs)
				 * 48		4		Unknown offset; 96 (0x0060) to start of URL record?
				 * 52		4		The location offset; 104 (0x0068) to start of URL record
				 * 56		1		Cache directory index (0=first, 0xFE=cookie?, 0xFF=.cdf?
				 * 57		1		Unknown (0x00, 0x01, 0x02, 0x03)
				 * 58		2		Unknown (0x1010) does not change?
				 * 60		4		The filename offset from start of URL record, or 0 if not set
				 * 64		4		Cache entry flags
				 * 68		4		The data offset from start of URL record or 0 if not set
				 * 72		4		The data size
				 * 76		4		Unknown, empty
				 * 80		4		Last checked date and time or 0
				 * 84		4		Number of hits
				 * 88		4		Unknown, empty, use count used in memory?)
				 * 92		4		Unknown, last cache synchronization date and time or 0?
				 * 96		4		Unknown, (8 byte aligned)
				 * 100		4		Unknown, uninitialized
				 * 104				ASCII string with a NUL-char, URL encoded
				 * (filnam offset) ... The filename in cache directory, ASCII nul-term
				 * (data offset) (data size) Data...
				 * ...		?	Uninitialized
				 */

				lookup.Sort();
				Console.WriteLine();
				Console.WriteLine("DATABASE RECORDS");
				Console.WriteLine();

				DateTime dttm;

				foreach (int offset in lookup)
				{
					stream.Position = offset;

					// will have at least one block
					buffer = new byte[BlockSize];
					if (stream.Read(buffer, 0, BlockSize) == BlockSize)
					{
						Console.WriteLine();
						Console.WriteLine(offset.ToString("X"));

						signature = ASCIIEncoding.ASCII.GetString(buffer, 0, 4);
						Console.WriteLine("Signature..............: [" + signature + "]");

						number = BitConverter.ToInt32(buffer, 4);
						Console.WriteLine("Number of blocks.......: " + number);

						if (number >= 1)
						{
							// we need to read numBlocks-1 since we already have the first
							byte[] temp = new byte[BlockSize * number];
							buffer.CopyTo(temp, 0);

							// read remaining blocks
							for (int i = 1; i < number; i++)
							{
								stream.Read(temp, BlockSize * i, BlockSize);
							}

							buffer = temp;
						}

						dttm = DateTime.FromFileTime(BitConverter.ToInt64(buffer, 8));
						Console.WriteLine("Last accessed..........: " + dttm.ToString(DateFormat));

						dttm = DateTime.FromFileTime(BitConverter.ToInt64(buffer, 16));
						Console.WriteLine("Last modified..........: " + dttm.ToString(DateFormat));

						int lo = BitConverter.ToInt32(buffer, 32);
						int hi = BitConverter.ToInt32(buffer, 36);
						long filesize = lo + (hi << 32);
						Console.WriteLine("File size..............: " + filesize + " (" + (filesize / 1024) + "KB)");

						number = BitConverter.ToInt32(buffer, 52);
						Console.Write("Start of URL offset....: " + number);
						StringBuilder builder = new StringBuilder();
						while (buffer[number] != 0)
						{
							builder.Append((char)buffer[number++]);
						}
						Console.WriteLine(" [" + builder.ToString() + "]");

						number = BitConverter.ToInt32(buffer, 60);
						Console.Write("Start of filename......: " + number);
						builder = new StringBuilder();
						while (buffer[number] != 0)
						{
							builder.Append((char)buffer[number++]);
						}
						Console.WriteLine(" [" + builder.ToString() + "]");

						number = BitConverter.ToInt32(buffer, 64);
						Console.WriteLine("Cache flags............: " + number.ToString("X")
							+ " (" + ((CacheFlags)number).ToString() + ")");

						number = BitConverter.ToInt32(buffer, 84);
						Console.WriteLine("Hits...................: " + number);

						number = BitConverter.ToInt32(buffer, 68);
						Console.WriteLine("Data offset............: " + number);

						number = BitConverter.ToInt32(buffer, 72);
						Console.WriteLine("Data size..............: " + number);
					}
				}

				/*
				 * URL RECORD LAYOUT
				 * 
				 * OFFSET	SIZE	DESCRIPTION
				 * 0		4		The signature (“REDR”)
				 * 4		4		The number of blocks in redirected record
				 * 8		4		Unknown, Seems to be an offset into a HASH record
				 * 12		4		Unknown, Record hash of the corresponding URL record?
				 * 16		...		The location, ASCII string with a NUL-character
				 */

				Console.WriteLine();
				Console.WriteLine("REDIRECT RECORDS");
				Console.WriteLine();

				if (redirects.Count > 0)
				{
					redirects.Sort();

					foreach (int offset in redirects)
					{
						stream.Position = redirects[0];

						// will have at least one block
						buffer = new byte[BlockSize];
						if (stream.Read(buffer, 0, BlockSize) == BlockSize)
						{
							Console.WriteLine();
							Console.WriteLine(offset.ToString("X"));

							signature = ASCIIEncoding.ASCII.GetString(buffer, 0, 4);
							Console.WriteLine("Signature..............: [" + signature + "]");

							number = BitConverter.ToInt32(buffer, 4);
							Console.WriteLine("Number of blocks.......: " + number);

							if (number >= 1)
							{
								// we need to read numBlocks-1 since we already have the first
								byte[] temp = new byte[(BlockSize * number)];
								buffer.CopyTo(temp, 0);

								// read remaining blocks
								for (int i = 1; i < number; i++)
								{
									stream.Read(temp, BlockSize * i, BlockSize);
								}

								buffer = temp;
							}

							int i2 = 0;
							var builder = new StringBuilder();
							while (buffer[i2] != 0)
							{
								builder.Append((char)buffer[i2++]);
							}

							Console.WriteLine("Location...............: " + builder.ToString());
						}
					}
				}
				else
				{
					Console.WriteLine("none");
				}
			}
		}


		#region unit test stuff
		public MMFDemo ()
		{
			//
			// TODO: Add constructor logic here
			//
		}

		private TestContext testContextInstance;

		/// <summary>
		///Gets or sets the test context which provides
		///information about and functionality for the current test run.
		///</summary>
		public TestContext TestContext
		{
			get
			{
				return testContextInstance;
			}
			set
			{
				testContextInstance = value;
			}
		}

		//
		// You can use the following additional attributes as you write your tests:
		//
		// Use ClassInitialize to run code before running the first test in the class
		// [ClassInitialize()]
		// public static void MyClassInitialize(TestContext testContext) { }
		//
		// Use ClassCleanup to run code after all tests in a class have run
		// [ClassCleanup()]
		// public static void MyClassCleanup() { }
		//
		// Use TestInitialize to run code before running each test 
		// [TestInitialize()]
		// public void MyTestInitialize() { }
		//
		// Use TestCleanup to run code after each test has run
		// [TestCleanup()]
		// public void MyTestCleanup() { }
		//
		#endregion stuff
	}
}
